package org.heydr.example.crawler;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.heydr.Crawler;
import org.heydr.common.TaskEngine;
import org.heydr.common.http.StringUtils;
import org.heydr.common.http.URLUtil;
import org.heydr.data.ResourceEngine;
import org.heydr.entity.Page;
import org.heydr.entity.Site;
import org.heydr.entity.Url;

public class MyCrawler extends Crawler{
	
	private ResourceEngine resourceEngine;
	
	public MyCrawler(ResourceEngine resourceEngine) {
		this.resourceEngine = resourceEngine;
	}
	
	private Pattern pattern;
	
	@Override
	public Object parse(Url url,Page page,int urlType) {
		switch(urlType){
			case 1:
				Set<Url> urls = new HashSet<Url>();
				Site site = TaskEngine.getSite(url.getSite());
				if(null != site){
					urls.addAll(extractUrls(page.getContent(),site,url,1));
					urls.addAll(extractUrls(page.getContent(),site,url,2));
				}
				return urls;
			case 2:
				
				return null;
		}
		return null;
	}

	private Set<Url> extractUrls(String content,Site site,Url url,int type){
		if(type==1){
		}else{
		}
		Matcher listMatcher = pattern.matcher(content);
		Set<Url> set = new HashSet<Url>();
		while (listMatcher.find()) {
			String listUrl = listMatcher.group(1).trim();
			listUrl = URLUtil.completeUrl(listUrl, site.getSite());
			if (null != listUrl && !"".equals(listUrl)) {
				// 去重
				Url u = new Url();
				u.setUrl(listUrl);
				u.setType(type);
				u.setParentUrl(url.getUrl());
				u.setFailTimes(0);
				u.setStatus(0);
				String urlMd5 = StringUtils.md5(listUrl);
				u.setUrlMd5(urlMd5);
				u.setSite(site.getSite());
				set.add(u);
			}
		}
		return set;
	}
	
	@Override
	public void snatchFail(Url url,int urlType) {
		url.setStatus(-1);
	}

	@Override
	public void storeInfo(Object o, int urlType) {
		switch (urlType) {
		case 1:
			Set<Url> set = (Set<Url>)o;
			List<Url> list = new ArrayList<Url>();
			list.addAll(set);
			resourceEngine.storeUrls(list);
			break;
		case 2:
			break;
		}
		
	}


}